{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting jieba\n",
      "  Downloading https://files.pythonhosted.org/packages/c6/cb/18eeb235f833b726522d7ebed54f2278ce28ba9438e3135ab0278d9792a2/jieba-0.42.1.tar.gz (19.2MB)\n",
      "Building wheels for collected packages: jieba\n",
      "  Running setup.py bdist_wheel for jieba: started\n",
      "  Running setup.py bdist_wheel for jieba: finished with status 'done'\n",
      "  Stored in directory: C:\\Users\\Administrator\\AppData\\Local\\pip\\Cache\\wheels\\af\\e4\\8e\\5fdd61a6b45032936b8f9ae2044ab33e61577950ce8e0dec29\n",
      "Successfully built jieba\n",
      "Installing collected packages: jieba\n",
      "Successfully installed jieba-0.42.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "twisted 18.7.0 requires PyHamcrest>=1.9.0, which is not installed.\n",
      "You are using pip version 10.0.1, however version 24.0 is available.\n",
      "You should consider upgrading via the 'python -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gensim\n",
      "  Downloading https://files.pythonhosted.org/packages/8a/6f/a690547cb7089d4019465bfbfbbb8bea5b3e52969cd2d6005049e6678ec4/gensim-4.2.0-cp37-cp37m-win_amd64.whl (24.0MB)\n",
      "Collecting numpy>=1.17.0 (from gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/97/9f/da37cc4a188a1d5d203d65ab28d6504e17594b5342e0c1dc5610ee6f4535/numpy-1.21.6-cp37-cp37m-win_amd64.whl (14.0MB)\n",
      "Requirement already satisfied: scipy>=0.18.1 in c:\\users\\administrator\\anaconda3\\lib\\site-packages (from gensim) (1.1.0)\n",
      "Collecting smart-open>=1.8.1 (from gensim)\n",
      "  Using cached https://files.pythonhosted.org/packages/7a/18/9a8d9f01957aa1f8bbc5676d54c2e33102d247e146c1a3679d3bd5cc2e3a/smart_open-7.1.0-py3-none-any.whl\n",
      "Collecting Cython==0.29.28 (from gensim)\n",
      "  Using cached https://files.pythonhosted.org/packages/9f/79/311cfbca90332ab37ef8ea08f1af3266f20a9a0e7a1d652842db832226bb/Cython-0.29.28-py2.py3-none-any.whl\n",
      "Requirement already satisfied: wrapt in c:\\users\\administrator\\anaconda3\\lib\\site-packages (from smart-open>=1.8.1->gensim) (1.10.11)\n",
      "Installing collected packages: numpy, smart-open, Cython, gensim\n",
      "  Found existing installation: numpy 1.15.1\n",
      "    Uninstalling numpy-1.15.1:\n",
      "      Successfully uninstalled numpy-1.15.1\n",
      "  Found existing installation: Cython 0.28.5\n",
      "    Uninstalling Cython-0.28.5:\n",
      "      Successfully uninstalled Cython-0.28.5\n",
      "Successfully installed Cython-0.29.28 gensim-4.2.0 numpy-1.21.6 smart-open-7.1.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "twisted 18.7.0 requires PyHamcrest>=1.9.0, which is not installed.\n",
      "You are using pip version 10.0.1, however version 24.0 is available.\n",
      "You should consider upgrading via the 'python -m pip install --upgrade pip' command.\n"
     ]
    }
   ],
   "source": [
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.829 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'周瑜'的词向量为:\n",
      " [-0.30626273 -0.31298304  0.21788514  1.2171773  -0.08306663  0.3761746\n",
      " -0.18271248  0.6230101  -0.9516201   0.1353504   0.6469788  -0.8766151\n",
      " -0.42399982 -0.31708235 -0.05107222  0.00165814 -0.42538744 -0.08988342\n",
      " -0.60085225 -1.0557879 ]\n",
      "与'周瑜'相似度最高的10个词:\n",
      "[('孟获', 0.9265453219413757), ('孙策', 0.9187067747116089), ('陆逊', 0.9125537872314453), ('钟会', 0.911077618598938), ('孙夫人', 0.9085162281990051), ('吕布', 0.9077621698379517), ('邓艾', 0.9058491587638855), ('孟达', 0.8978295922279358), ('曹真', 0.8957856893539429), ('孙权', 0.8937253355979919)]\n",
      "'刘备'和'曹操'的相似度:0.8008544445037842\n",
      "在词'孙权/曹操/刘备/孙夫人'中，'刘备'与其他词不属于同一类\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "import re\n",
    "from gensim.models import Word2Vec\n",
    "with open(\"sanguo.txt\",encoding='utf-8') as f:\n",
    "    lines=[]\n",
    "    for line in f:\n",
    "        temp=jieba.lcut(line)\n",
    "        words=[]\n",
    "        for i in temp:\n",
    "            i = re.sub(\"[\\s+\\.\\!\\/_,$%^*(+\\\"\\'””《》]+|[+——！，。？、~@#￥%……&*（）：；‘]+\", \"\", i)\n",
    "            if len(i)>0:\n",
    "                words.append(i)\n",
    "        if len(words)>0:\n",
    "            lines.append(words)\n",
    "model=Word2Vec(lines,vector_size=20,window=2,min_count=3,epochs=7,negative=10,sg=1)\n",
    "print(\"'周瑜'的词向量为:\\n\",model.wv.get_vector('周瑜'))\n",
    "print(\"与'周瑜'相似度最高的10个词:\")\n",
    "print(model.wv.most_similar('周瑜',topn=10))\n",
    "print(\"'刘备'和'曹操'的相似度:{}\".format(model.wv.similarity('刘备','曹操')))\n",
    "words=\"孙权 曹操 刘备 孙夫人\"\n",
    "print(\"在词'孙权/曹操/刘备/孙夫人'中，'{}'与其他词不属于同一类\".format(model.wv.doesnt_match(words.split())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "from gensim.models.doc2vec import Doc2Vec\n",
    "TaggededDocument=gensim.models.doc2vec.TaggedDocument\n",
    "def get_data():\n",
    "    with open(\"train.txt\",'r',encoding='utf-8')as f:\n",
    "        docs=f.readlines()\n",
    "        train_data=[]\n",
    "        for i,text in enumerate(docs):\n",
    "            word_list=text.split(' ')\n",
    "            word_list[len(word_list)-1]=word_list[len(word_list)-1].strip\n",
    "            document=TaggededDocument(word_list,tags=[i])\n",
    "            train_data.append(document)\n",
    "        return train_data\n",
    "def train_model(x_train):\n",
    "        model_dm=Doc2Vec(x_train,min_count=1,window=3,vector_size=20,negative=5,workers=4,dm=1)\n",
    "        model_dm.train(x_train,total_examples=model_dm.corpus_count,epochs=70)\n",
    "        model_dm.save(\"model_doc2vec\")\n",
    "        return model_dm\n",
    "def test():\n",
    "        model_dm=Doc2Vec.load(\"model_doc2vec\")\n",
    "        test_text=['科学','教育','是','难搞','的']\n",
    "        inferred_vector_dm=model_dm.infer_vector(test_text)\n",
    "        sims=model_dm.docvecs.most_similar([inferred_vector_dm],topn=10)\n",
    "        return sims\n",
    "if __name__ =='__main__':\n",
    "        train_data=get_data()\n",
    "        model_dm=train_model(train_data)\n",
    "        sims=test()\n",
    "        print(\"相似文本、相似度和文本中词的数量:\")\n",
    "        for count,sim in sims:\n",
    "            sentence=train_data[count]\n",
    "            words=''\n",
    "            for word in sentence[0]:\n",
    "                words=words+word+''\n",
    "                print(words,sim,len(sentence[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
